{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rtox72csOQUN"
   },
   "source": [
    "# DeepMatch 样例代码\n",
    "- https://github.com/shenweichen/DeepMatch\n",
    "- https://deepmatch.readthedocs.io/en/latest/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bTWHz-heMkyw"
   },
   "source": [
    "# 下载movielens-1M数据 安装依赖包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yTl6d6jO1oqf",
    "outputId": "ee7303f1-8970-4726-a9f1-368798077228"
   },
   "outputs": [],
   "source": [
    "! wget http://files.grouplens.org/datasets/movielens/ml-1m.zip -O ./ml-1m.zip \n",
    "! wget https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py -O preprocess.py\n",
    "! unzip -o ml-1m.zip \n",
    "! pip uninstall -y -q tensorflow\n",
    "! pip install -q tensorflow-gpu==2.5.0\n",
    "! pip install -q deepmatch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p9UxNHuPMuW2"
   },
   "source": [
    "# 导入需要的库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "C_ZR6gzp1E2N"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:469: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:470: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:471: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:472: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:473: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:476: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Users/swc/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from deepctr.feature_column import SparseFeat, VarLenSparseFeat\n",
    "from preprocess import gen_data_set, gen_model_input\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tensorflow.python.keras import backend as K\n",
    "from tensorflow.python.keras.models import Model\n",
    "\n",
    "from deepmatch.models import *\n",
    "from deepmatch.utils import sampledsoftmaxloss, NegativeSampler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fQq6O9XAMzPF"
   },
   "source": [
    "# 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lcO29zFb21Od",
    "outputId": "bfeed1ac-99f2-425f-dda6-10b83be721fe"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  after removing the cwd from sys.path.\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:8: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "data_path = \"./\"\n",
    "\n",
    "unames = ['user_id','gender','age','occupation','zip']\n",
    "user = pd.read_csv(data_path+'ml-1m/users.dat',sep='::',header=None,names=unames)\n",
    "rnames = ['user_id','movie_id','rating','timestamp']\n",
    "ratings = pd.read_csv(data_path+'ml-1m/ratings.dat',sep='::',header=None,names=rnames)\n",
    "mnames = ['movie_id','title','genres']\n",
    "movies = pd.read_csv(data_path+'ml-1m/movies.dat',sep='::',header=None,names=mnames,encoding=\"unicode_escape\")\n",
    "movies['genres'] = list(map(lambda x: x.split('|')[0], movies['genres'].values))\n",
    "\n",
    "data = pd.merge(pd.merge(ratings,movies),user)#.iloc[:10000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L0yCWxQxM3se"
   },
   "source": [
    "# 构建特征列，训练模型，导出embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BMOvk_de2ML3",
    "outputId": "962afe1c-d387-4345-861f-e9b974a0b495"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6040/6040 [00:11<00:00, 508.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 8\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "make sure the activation function use training flag properly call() got an unexpected keyword argument 'training'\n",
      "Epoch 1/20\n",
      "988129/988129 [==============================] - 111s - loss: 5.1306   \n",
      "Epoch 2/20\n",
      "988129/988129 [==============================] - 110s - loss: 4.4118   \n",
      "Epoch 3/20\n",
      "988129/988129 [==============================] - 111s - loss: 4.1463   \n",
      "Epoch 4/20\n",
      "988129/988129 [==============================] - 116s - loss: 3.9994   \n",
      "Epoch 5/20\n",
      "988129/988129 [==============================] - 115s - loss: 3.8970   \n",
      "Epoch 6/20\n",
      "988129/988129 [==============================] - 124s - loss: 3.8210   \n",
      "Epoch 7/20\n",
      "988129/988129 [==============================] - 117s - loss: 3.7645   \n",
      "Epoch 8/20\n",
      "988129/988129 [==============================] - 112s - loss: 3.7182   \n",
      "Epoch 9/20\n",
      "988129/988129 [==============================] - 112s - loss: 3.6805   \n",
      "Epoch 10/20\n",
      "988129/988129 [==============================] - 111s - loss: 3.6507   \n",
      "Epoch 11/20\n",
      "988129/988129 [==============================] - 137s - loss: 3.6256   \n",
      "Epoch 12/20\n",
      "988129/988129 [==============================] - 132s - loss: 3.6034   \n",
      "Epoch 13/20\n",
      "988129/988129 [==============================] - 118s - loss: 3.5852   \n",
      "Epoch 14/20\n",
      "988129/988129 [==============================] - 108s - loss: 3.5706   \n",
      "Epoch 15/20\n",
      "988129/988129 [==============================] - 108s - loss: 3.5567   \n",
      "Epoch 16/20\n",
      "988129/988129 [==============================] - 109s - loss: 3.5453   \n",
      "Epoch 17/20\n",
      "988129/988129 [==============================] - 148s - loss: 3.5338   \n",
      "Epoch 18/20\n",
      "988129/988129 [==============================] - 123s - loss: 3.5255   \n",
      "Epoch 19/20\n",
      "988129/988129 [==============================] - 296s - loss: 3.5165   \n",
      "Epoch 20/20\n",
      "988129/988129 [==============================] - 121s - loss: 3.5099   \n",
      "(6040, 2, 32)\n",
      "(3706, 32)\n"
     ]
    }
   ],
   "source": [
    "#data = pd.read_csvdata = pd.read_csv(\"./movielens_sample.txt\")\n",
    "sparse_features = [\"movie_id\", \"user_id\",\n",
    "                    \"gender\", \"age\", \"occupation\", \"zip\", \"genres\"]\n",
    "SEQ_LEN = 50\n",
    "negsample = 0\n",
    "\n",
    "# 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input`\n",
    "\n",
    "feature_max_idx = {}\n",
    "for feature in sparse_features:\n",
    "    lbe = LabelEncoder()\n",
    "    data[feature] = lbe.fit_transform(data[feature]) + 1\n",
    "    feature_max_idx[feature] = data[feature].max() + 1\n",
    "\n",
    "user_profile = data[[\"user_id\", \"gender\", \"age\", \"occupation\", \"zip\"]].drop_duplicates('user_id')\n",
    "\n",
    "item_profile = data[[\"movie_id\"]].drop_duplicates('movie_id')\n",
    "\n",
    "user_profile.set_index(\"user_id\", inplace=True)\n",
    "\n",
    "user_item_list = data.groupby(\"user_id\")['movie_id'].apply(list)\n",
    "\n",
    "train_set, test_set = gen_data_set(data, SEQ_LEN, negsample)\n",
    "\n",
    "train_model_input, train_label = gen_model_input(train_set, user_profile, SEQ_LEN)\n",
    "test_model_input, test_label = gen_model_input(test_set, user_profile, SEQ_LEN)\n",
    "\n",
    "# 2.count #unique features for each sparse field and generate feature config for sequence feature\n",
    "\n",
    "embedding_dim = 32\n",
    "\n",
    "user_feature_columns = [SparseFeat('user_id', feature_max_idx['user_id'], 16),\n",
    "                        SparseFeat(\"gender\", feature_max_idx['gender'], 16),\n",
    "                        SparseFeat(\"age\", feature_max_idx['age'], 16),\n",
    "                        SparseFeat(\"occupation\", feature_max_idx['occupation'], 16),\n",
    "                        SparseFeat(\"zip\", feature_max_idx['zip'], 16),\n",
    "                        VarLenSparseFeat(SparseFeat('hist_movie_id', feature_max_idx['movie_id'], embedding_dim,\n",
    "                                                    embedding_name=\"movie_id\"), SEQ_LEN, 'mean', 'hist_len'),\n",
    "                        VarLenSparseFeat(SparseFeat('hist_genres', feature_max_idx['genres'], embedding_dim,\n",
    "                                                   embedding_name=\"genres\"), SEQ_LEN, 'mean', 'hist_len'),\n",
    "                        ]\n",
    "\n",
    "item_feature_columns = [SparseFeat('movie_id', feature_max_idx['movie_id'], embedding_dim)]\n",
    "\n",
    "from collections import Counter\n",
    "train_counter = Counter(train_model_input['movie_id'])\n",
    "item_count = [train_counter.get(i,0) for i in range(item_feature_columns[0].vocabulary_size)]\n",
    "sampler_config = NegativeSampler('frequency',num_sampled=255,item_name=\"movie_id\",item_count=item_count)\n",
    "\n",
    "# 3.Define Model and train\n",
    "\n",
    "import tensorflow as tf\n",
    "if tf.__version__ >= '2.0.0':\n",
    "    tf.compat.v1.disable_eager_execution()\n",
    "else:\n",
    "    K.set_learning_phase(True)\n",
    "    \n",
    "#model = YoutubeDNN(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(128,64, embedding_dim), sampler_config=sampler_config)\n",
    "model = ComiRec(user_feature_columns,item_feature_columns,k_max=2, user_dnn_hidden_units=(128,64, embedding_dim), sampler_config=sampler_config)\n",
    "\n",
    "model.compile(optimizer=\"adam\", loss=sampledsoftmaxloss)\n",
    "\n",
    "history = model.fit(train_model_input, train_label,  # train_label,\n",
    "                    batch_size=512, epochs=20, verbose=1, validation_split=0.0, )\n",
    "\n",
    "# 4. Generate user features for testing and full item features for retrieval\n",
    "test_user_model_input = test_model_input\n",
    "all_item_model_input = {\"movie_id\": item_profile['movie_id'].values,}\n",
    "\n",
    "user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding)\n",
    "item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding)\n",
    "\n",
    "user_embs = user_embedding_model.predict(test_user_model_input, batch_size=2 ** 12)\n",
    "# user_embs = user_embs[:, i, :]  # i in [0,k_max) if MIND\n",
    "item_embs = item_embedding_model.predict(all_item_model_input, batch_size=2 ** 12)\n",
    "\n",
    "print(user_embs.shape)\n",
    "print(item_embs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "w_G3KWslKmJo"
   },
   "source": [
    "# 使用faiss进行ANN查找并评估结果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5SvyQLNVKkcs"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "j2ZNYNBOOqrN",
    "outputId": "2eec5e82-2d2b-4fe0-9b83-2a74a4dc52ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.7/dist-packages (1.7.2)\n"
     ]
    }
   ],
   "source": [
    "! pip install faiss-cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6TY1l27iJU8U",
    "outputId": "5a8ccdd3-af70-4c48-b859-84c4befddfdd"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6040/6040 [00:00<00:00, 6105.92it/s]\n",
      "100%|██████████| 6040/6040 [00:01<00:00, 5487.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "recall 0.43642384105960264\n",
      "hr 0.43642384105960264\n"
     ]
    }
   ],
   "source": [
    "import heapq\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import faiss\n",
    "from deepmatch.utils import recall_N\n",
    "\n",
    "k_max = 2\n",
    "topN = 50\n",
    "test_true_label = {line[0]: [line[1]] for line in test_set}\n",
    "\n",
    "index = faiss.IndexFlatIP(embedding_dim)\n",
    "# faiss.normalize_L2(item_embs)\n",
    "index.add(item_embs)\n",
    "# faiss.normalize_L2(user_embs)\n",
    "\n",
    "if len(user_embs.shape) == 2:  # multi interests model's shape = 3 (MIND,ComiRec)\n",
    "    user_embs = np.expand_dims(user_embs, axis=1)\n",
    "\n",
    "score_dict = defaultdict(dict)\n",
    "for k in range(k_max):\n",
    "    user_emb = user_embs[:, k, :]\n",
    "    D, I = index.search(np.ascontiguousarray(user_emb), topN)\n",
    "    for i, uid in tqdm(enumerate(test_user_model_input['user_id']), total=len(test_user_model_input['user_id'])):\n",
    "        if np.abs(user_emb[i]).max() < 1e-8:\n",
    "            continue\n",
    "        for score, itemid in zip(D[i], I[i]):\n",
    "            score_dict[uid][itemid] = max(score, score_dict[uid].get(itemid, float(\"-inf\")))\n",
    "\n",
    "s = []\n",
    "hit = 0\n",
    "for i, uid in enumerate(test_user_model_input['user_id']):\n",
    "    pred = [item_profile['movie_id'].values[x[0]] for x in\n",
    "            heapq.nlargest(topN, score_dict[uid].items(), key=lambda x: x[1])]\n",
    "    filter_item = None\n",
    "    recall_score = recall_N(test_true_label[uid], pred, N=topN)\n",
    "    s.append(recall_score)\n",
    "    if test_true_label[uid] in pred:\n",
    "        hit += 1\n",
    "\n",
    "print(\"recall\", np.mean(s))\n",
    "print(\"hr\", hit / len(test_user_model_input['user_id']))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "colab_MovieLen1M_YoutubeDNN.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
